{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import numpy\n",
    "import pandas as pd\n",
    "\n",
    "from adfilter.model import AdPredictor\n",
    "\n",
    "allAdText = open(r\"./data/ad.txt\", 'r', encoding='utf-8').read()\n",
    "allAdTextArray = allAdText.split('\\n\\n')\n",
    "\n",
    "allNormalText = open(r\"./data/normal.txt\", 'r', encoding='utf-8').read()\n",
    "allNormalTextArray = allNormalText.split('\\n')\n",
    "\n",
    "# df = pd.read_csv(\"./data/dataset.csv\", encoding='utf-8')\n",
    "# df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\Kenvix\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 1.035 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "data": {
      "text/plain": "         一下  一下 大学  一下 这人   一两  一两 一两年  一两 一两秒  一两年  一两年 两年  一两秒  一两秒 两秒  ...  \\\n0  0.000000    0.0    0.0  0.0     0.0     0.0  0.0     0.0  0.0     0.0  ...   \n1  0.000000    0.0    0.0  0.0     0.0     0.0  0.0     0.0  0.0     0.0  ...   \n2  0.102528    0.0    0.0  0.0     0.0     0.0  0.0     0.0  0.0     0.0  ...   \n3  0.000000    0.0    0.0  0.0     0.0     0.0  0.0     0.0  0.0     0.0  ...   \n4  0.000000    0.0    0.0  0.0     0.0     0.0  0.0     0.0  0.0     0.0  ...   \n\n     鸡  鸡 双击   麻油  麻油 问题    黑  黑 苹果   黑屏  黑屏 亮  黑屏 双击  黑屏 过重  \n0  0.0   0.0  0.0    0.0  0.0   0.0  0.0   0.0    0.0    0.0  \n1  0.0   0.0  0.0    0.0  0.0   0.0  0.0   0.0    0.0    0.0  \n2  0.0   0.0  0.0    0.0  0.0   0.0  0.0   0.0    0.0    0.0  \n3  0.0   0.0  0.0    0.0  0.0   0.0  0.0   0.0    0.0    0.0  \n4  0.0   0.0  0.0    0.0  0.0   0.0  0.0   0.0    0.0    0.0  \n\n[5 rows x 1992 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>一下</th>\n      <th>一下 大学</th>\n      <th>一下 这人</th>\n      <th>一两</th>\n      <th>一两 一两年</th>\n      <th>一两 一两秒</th>\n      <th>一两年</th>\n      <th>一两年 两年</th>\n      <th>一两秒</th>\n      <th>一两秒 两秒</th>\n      <th>...</th>\n      <th>鸡</th>\n      <th>鸡 双击</th>\n      <th>麻油</th>\n      <th>麻油 问题</th>\n      <th>黑</th>\n      <th>黑 苹果</th>\n      <th>黑屏</th>\n      <th>黑屏 亮</th>\n      <th>黑屏 双击</th>\n      <th>黑屏 过重</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.102528</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 1992 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor = AdPredictor()\n",
    "splitNormalWords = [list(predictor.splitWords(ad)) for ad in allNormalTextArray]\n",
    "normalDocument = [\" \".join(sent0) for sent0 in splitNormalWords]\n",
    "\n",
    "splitAdWords = [list(predictor.splitWords(ad)) for ad in allAdTextArray]\n",
    "adDocument = [\" \".join(sent0) for sent0 in splitAdWords]\n",
    "\n",
    "combined = []\n",
    "combined.extend(adDocument)\n",
    "combined.extend(normalDocument)\n",
    "\n",
    "\n",
    "textMatrix = predictor.transformTextToSparseMatrix(combined)\n",
    "textMatrix.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ad len: 24 Normal len: 185\n",
      "Train Num:  188 Test Num:  21\n",
      "Test Acc 1.0\n",
      "Saving check point \n"
     ]
    }
   ],
   "source": [
    "features = pd.DataFrame(textMatrix.apply(sum,axis=0))\n",
    "extractedfeatures = [features.index[i] for i in range(features.shape[0]) if features.iloc[i,0]>5]\n",
    "# textMatrix = textMatrix[extractedfeatures]\n",
    "\n",
    "print(\"Ad len:\", len(adDocument), \"Normal len:\", len(normalDocument))\n",
    "\n",
    "y = []\n",
    "y.extend(numpy.ones(len(adDocument)))\n",
    "y.extend(numpy.zeros(len(normalDocument)))\n",
    "\n",
    "predictor.train(textMatrix, y)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unhashable type: 'numpy.ndarray'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-6-45c04b686f32>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      2\u001B[0m \u001B[0mtext2\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34m\"java垃圾！C垃圾！计算机二级垃圾！PS学姐biss！资料墙biss！👴精通CAD!！\"\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      3\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 4\u001B[1;33m \u001B[0mprint\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpredictor\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpredict_ad\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtext1\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mpredictor\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpredict_ad\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtext2\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32mC:\\Work-Station\\Python\\BotBackend\\PyBackend\\adfilter\\model.py\u001B[0m in \u001B[0;36mpredict_ad\u001B[1;34m(self, text)\u001B[0m\n\u001B[0;32m     88\u001B[0m         \u001B[0mvector\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mvectorizer\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mtransform\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mdocument\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     89\u001B[0m         \u001B[0mresult\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mDataFrame\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mvector\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mtoarray\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 90\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget_type_string\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mclf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpredict\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mresult\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32mC:\\Work-Station\\Python\\BotBackend\\PyBackend\\adfilter\\model.py\u001B[0m in \u001B[0;36mget_type_string\u001B[1;34m(self, type_int)\u001B[0m\n\u001B[0;32m     81\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     82\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0mget_type_string\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mtype_int\u001B[0m\u001B[1;33m:\u001B[0m \u001B[0mint\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m->\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 83\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mtypes\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtype_int\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     84\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     85\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0mpredict_ad\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mtext\u001B[0m\u001B[1;33m:\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m->\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mTypeError\u001B[0m: unhashable type: 'numpy.ndarray'"
     ]
    }
   ],
   "source": [
    "text1 = \"这个是咱们学校的学习墙开学准备（基本包含各学科）技能提升（计算机二级  word  ppt等）各种考证资料（考研 四六级  二级  教资 单招 会计 专升本等）基本都有的，抗疫时期闲着也是闲着 需要的加墙墙就好啦 \"\n",
    "text2 = \"java垃圾！C垃圾！计算机二级垃圾！PS学姐biss！资料墙biss！👴精通CAD!！\"\n",
    "\n",
    "print(predictor.predict_ad(text1), predictor.predict_ad(text2))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}